import regex

import scrapy

from MySpider.items.wqlgameitems.wqlgamedetailitem import WQLGameDetailItem
import requests
from MySpider.mymethod import get_info_xpath_then_patten
from MySpider.spiders.wqlgamespiders import spider_parameters


class WqlGameDetailSpider(scrapy.Spider):
    name = "WqlGameDetailSpider"
    allowed_domains = spider_parameters['allowed_domains'].split("|")
    insert_sql = spider_parameters['detail_insert_sql']
    start_urls = spider_parameters['detail_start_urls']
    table_fields = spider_parameters['detail_table_fields']
    use_cookie = True
    cookies = spider_parameters['cookies']
    # 以下为核心代码，我把它抽象为网虫爬取三要素
    # 1.每个元素的xpath用来定位信息
    # 2.regex_patten用来修剪信息
    # 3.item_info_map用来链接source，xpath，regex_patten
    # 基本上以上三个确定了，爬虫就可以工作了，当然除了一些特殊的，例如需要重定向才能爬取的信息
    # 还有一个特殊处理，在C#中可以使用委托实现，有资料提示python中使用装饰器来实现，但是我还没有头绪

    # path selector
    img_xpath = '//div[@class="content-area"]//img[@src and @class="img-fluid wp-post-image"]/@src'
    title_xpath = '//article[@id]/div/header/h1/text()'
    intro_xpath = '//article[@id]/div/div[2]/div/p/text()'
    baidu_url_xpath = '//div[@id="rizhuti_v2_shop_down-2"]//div[@class="btn-group btn-block mt-2"][1]/a[@href]/@href'
    game_pass_xpath = '//*[@id="rizhuti_v2_shop_down-2"]//button[@title="点击复制密码"]/@data-clipboard-text'
    # 正则筛选
    url_id_patten = regex.compile(r'(?<=https://www.688qf.com/)\w+')
    baidu_url_patten = regex.compile(r'(?<=pan.baidu.com/s/).*')
    # 前置不定式的需要引用regex而不是re，
    game_pass_patten = regex.compile(r'(?<=\s密码：|\s安装码：|\s解压密码：)\w+')

    # 提取地图，按照[source,xpath,patten]顺序排列
    item_info_map = [['url_id', 'response.url', 'None', 'self.url_id_patten'],
                     ['img', 'response', 'self.img_xpath', 'None'],
                     ['title', 'response', 'self.title_xpath', 'None'],
                     ['intro', 'response', 'self.intro_xpath', 'None'],
                     ['game_pass', 'response', 'self.game_pass_xpath', 'self.game_pass_patten']
                     ]

    def parse(self, response, **kwargs):
        item = WQLGameDetailItem()
        print("\n正在爬取网页%s........" % response.url)

        # 一个item元素的取得，由三个部分决定——source，xpath，regex_patten
        for m in self.item_info_map:
            item[m[0]] = get_info_xpath_then_patten(eval(m[1]), eval(m[2]), eval(m[3]))

        ''' 
        以下代码被上面for循环代码代替
        item['url_id'] = get_info_xpath_then_patten(response.url, patten=self.url_id_patten)
        item['img'] = get_info_xpath_then_patten(response, self.img_xpath)
        item['title'] = get_info_xpath_then_patten(response, self.title_xpath)
        item['intro'] = get_info_xpath_then_patten(response, self.intro_xpath)
        item['game_pass'] = get_info_xpath_then_patten(response, self.game_pass_xpath,
                                                       self.game_pass_patten)
        '''
        # 这个特殊，需要重定向后取得实际地址
        baidu_url = response.xpath(self.baidu_url_xpath).extract_first()
        baidu_url = requests.get(url=baidu_url, cookies=self.cookies, allow_redirects=False).headers.get('Location')
        item['baidu_url'] = self.baidu_url_patten.search(baidu_url).group(0)
        print("解析完成")
        yield item
